import numpy as np
import argparse
import torch
from copy import deepcopy
import math
from option_critic import OptionCriticFeatures
from option_critic import critic_loss as critic_loss_fn
from option_critic import actor_loss as actor_loss_fn

from experience_replay import ReplayBuffer
from utils import make_env, to_tensor
from logger import Logger

import matplotlib.pyplot as plt
import time

import os
os.environ["KMP_DUPLICATE_LIB_OK"]="TRUE"

parser = argparse.ArgumentParser(description="Option Critic PyTorch")
parser.add_argument('--env', default='ABR', help='ROM to run')
parser.add_argument('--optimal-eps', type=float, default=0.05, help='Epsilon when playing optimally')
parser.add_argument('--frame-skip', default=4, type=int, help='Every how many frames to process')
parser.add_argument('--learning-rate',type=float, default=.0005, help='Learning rate')
parser.add_argument('--gamma', type=float, default=.99, help='Discount rate')
parser.add_argument('--epsilon-start',  type=float, default=1.0, help=('Starting value for epsilon.'))
parser.add_argument('--epsilon-min', type=float, default=.1, help='Minimum epsilon.')
parser.add_argument('--epsilon-decay', type=float, default=1e5, help=('Number of steps to minimum epsilon.'))# 20000
parser.add_argument('--max-history', type=int, default=10000, help=('Maximum number of steps stored in replay'))
parser.add_argument('--batch-size', type=int, default=128, help='Batch size.') # 32
parser.add_argument('--freeze-interval', type=int, default=200, help=('Interval between target freezes.'))
parser.add_argument('--update-frequency', type=int, default=4, help=('Number of actions before each SGD update.'))
parser.add_argument('--termination-reg', type=float, default=0.01, help=('Regularization to decrease termination prob.'))
parser.add_argument('--entropy-reg', type=float, default=0.01, help=('Regularization to increase policy entropy.'))
parser.add_argument('--num-options', type=int, default=2, help=('Number of options to create.'))
parser.add_argument('--temp', type=float, default=1, help='Action distribution softmax tempurature param.')

parser.add_argument('--max_steps_ep', type=int, default=300, help='number of maximum steps per episode.') # 18000
parser.add_argument('--max_steps_total', type=int, default=int(3e4), help='number of maximum steps to take.') # 4e6  bout 4 million
parser.add_argument('--cuda', type=bool, default=True, help='Enable CUDA training (recommended if possible).')
parser.add_argument('--seed', type=int, default=0, help='Random seed for numpy, torch, random.')
parser.add_argument('--logdir', type=str, default='runs', help='Directory for logging statistics')
parser.add_argument('--exp', type=str, default=None, help='optional experiment name')
parser.add_argument('--switch-goal', type=bool, default=False, help='switch goal after 2k eps')

def run(args):
    env, is_atari = make_env(args.env)
    # option_critic = OptionCriticConv if is_atari else OptionCriticFeatures
    option_critic = OptionCriticFeatures

    device = torch.device('cuda' if torch.cuda.is_available() and args.cuda else 'cpu')
    print(device)
    

    option_critic = option_critic(
        in_features=env.observation_space.n, # 14
        num_actions=env.action_space.n, # 4096
        num_options=args.num_options,
        temperature=args.temp,
        eps_start=args.epsilon_start,
        eps_min=args.epsilon_min,
        eps_decay=args.epsilon_decay,
        eps_test=args.optimal_eps,
        device=device
    )

    # Create a prime network for more stable Q values
    option_critic_prime = deepcopy(option_critic)
    
    optim = torch.optim.SGD(option_critic.parameters(), lr=args.learning_rate)

    np.random.seed(args.seed)
    torch.manual_seed(args.seed)
    env.seed(args.seed)

    env.ep_steps_max = args.max_steps_ep

    buffer = ReplayBuffer(capacity=args.max_history, seed=args.seed) # 生成一个容器做buffer，存下数据
    logger = Logger(logdir=args.logdir, run_name=f"{OptionCriticFeatures.__name__}-{args.env}-{args.exp}-{time.ctime()}")
    MODEL_PATH = "models/" + f"{args.env}-{args.exp}-{time.ctime()}.pth"
    MODEL_PATH = MODEL_PATH.replace(':', '_')
    steps = 0 



    # 要存的数据  episode为单位
    latency_epi = []
    data_amount_epi = []
    rebuffer_epi = []
    bitrate_epi = []
    smoothness_epi = []
    reward_list = []
    rebuffer_time_epi = []
    FoV_inside_bitrate_epi = []
    FoV_adjacency_bitrate_epi = []
    FoV_outside_bitrate_epi = []
    smoothness_inside_FoV = []
    smoothness_adjacency_FoV = []
    smoothness_outside_FoV = []
    bitrate_level_number_epi = []

    option_list = []
    option_bitrate_list = []
    option_smoothness_list = []
    option_rebuffer_list = []

    option_one_length_list = []
    option_one_selection_list = []
    option_one_bitrate_list = []
    option_one_smoothness_list = []
    option_one_rebuffer_list = []

    DP_rebuffer_epi = []
    DP_bitrate_epi = []
    DP_smoothness_epi = []
    DP_reward_list = []
    DP_rebuffer_time_epi = []
    DP_option_list = []
    DP_FoV_inside_bitrate_epi = []
    DP_FoV_adjacency_bitrate_epi = []
    DP_FoV_outside_bitrate_epi = []
    DP_smoothness_inside_FoV = []
    DP_smoothness_adjacency_FoV = []
    DP_smoothness_outside_FoV = []
    DP_bitrate_level_number_epi = []

    HB_rebuffer_epi = []
    HB_bitrate_epi = []
    HB_smoothness_epi = []
    HB_reward_list = []
    HB_rebuffer_time_epi = []
    HB_option_list = []
    HB_FoV_inside_bitrate_epi = []
    HB_FoV_adjacency_bitrate_epi = []
    HB_FoV_outside_bitrate_epi = []
    HB_smoothness_inside_FoV = []
    HB_smoothness_adjacency_FoV = []
    HB_smoothness_outside_FoV = []
    HB_bitrate_level_number_epi = []

    LB_rebuffer_epi = []
    LB_bitrate_epi = []
    LB_smoothness_epi = []
    LB_reward_list = []
    LB_rebuffer_time_epi = []
    LB_option_list = []
    LB_FoV_inside_bitrate_epi = []
    LB_FoV_adjacency_bitrate_epi = []
    LB_FoV_outside_bitrate_epi = []
    LB_smoothness_inside_FoV = []
    LB_smoothness_adjacency_FoV = []
    LB_smoothness_outside_FoV = []
    LB_bitrate_level_number_epi = []

    Buffer_rebuffer_epi = []
    Buffer_bitrate_epi = []
    Buffer_smoothness_epi = []
    Buffer_reward_list = []
    Buffer_rebuffer_time_epi = []
    Buffer_option_list = []
    Buffer_FoV_inside_bitrate_epi = []
    Buffer_FoV_adjacency_bitrate_epi = []
    Buffer_FoV_outside_bitrate_epi = []
    Buffer_smoothness_inside_FoV = []
    Buffer_smoothness_adjacency_FoV = []
    Buffer_smoothness_outside_FoV = []
    Buffer_bitrate_level_number_epi = []

    Throughput_rebuffer_epi = []
    Throughput_bitrate_epi = []
    Throughput_smoothness_epi = []
    Throughput_reward_list = []
    Throughput_rebuffer_time_epi = []
    Throughput_option_list = []
    Throughput_FoV_inside_bitrate_epi = []
    Throughput_FoV_adjacency_bitrate_epi = []
    Throughput_FoV_outside_bitrate_epi = []
    Throughput_smoothness_inside_FoV = []
    Throughput_smoothness_adjacency_FoV = []
    Throughput_smoothness_outside_FoV = []
    Throughput_bitrate_level_number_epi = []

    BOLA_rebuffer_epi = []
    BOLA_bitrate_epi = []
    BOLA_smoothness_epi = []
    BOLA_reward_list = []
    BOLA_rebuffer_time_epi = []
    BOLA_option_list = []
    BOLA_FoV_inside_bitrate_epi = []
    BOLA_FoV_adjacency_bitrate_epi = []
    BOLA_FoV_outside_bitrate_epi = []
    BOLA_smoothness_inside_FoV = []
    BOLA_smoothness_adjacency_FoV = []
    BOLA_smoothness_outside_FoV = []

    while steps < args.max_steps_total:
        # print(steps)
        rewards = 0; option_lengths = {opt:[] for opt in range(args.num_options)}

        obs = env.reset()

        state = option_critic.get_state(to_tensor(obs))
        # 返回最大value的option
        greedy_option  = option_critic.greedy_option(state)
        current_option = 0   

        # 要存的数据 step为单位
        latency_step = []
        data_amount_step = []
        rebuffer_step = []
        bitrate_step = []
        smoothness_step = []
        rebuffer_time_step = []
        option_step = []
        FoV_inside_bitrate_step = []
        FoV_adjacency_bitrate_step = []
        FoV_outside_bitrate_step = []
        smoothness_inside_FoV_step = []
        smoothness_adjacency_FoV_step = []
        smoothness_outside_FoV_step = []
        bitrate_level_number_step = []

        DP_rebuffer_step = []
        DP_bitrate_step = []
        DP_smoothness_step = []
        DP_reward_step = 0
        DP_rebuffer_time_step = []
        DP_FoV_inside_bitrate_step = []
        DP_FoV_adjacency_bitrate_step = []
        DP_FoV_outside_bitrate_step = []
        DP_smoothness_inside_FoV_step = []
        DP_smoothness_adjacency_FoV_step = []
        DP_smoothness_outside_FoV_step = []
        DP_bitrate_level_number_step = []

        HB_rebuffer_step = []
        HB_bitrate_step = []
        HB_smoothness_step = []
        HB_reward_step = 0
        HB_rebuffer_time_step = []
        HB_FoV_inside_bitrate_step = []
        HB_FoV_adjacency_bitrate_step = []
        HB_FoV_outside_bitrate_step = []
        HB_smoothness_inside_FoV_step = []
        HB_smoothness_adjacency_FoV_step = []
        HB_smoothness_outside_FoV_step = []
        HB_bitrate_level_number_step = []

        LB_rebuffer_step = []
        LB_bitrate_step = []
        LB_smoothness_step = []
        LB_reward_step = 0
        LB_rebuffer_time_step = []
        LB_FoV_inside_bitrate_step = []
        LB_FoV_adjacency_bitrate_step = []
        LB_FoV_outside_bitrate_step = []
        LB_smoothness_inside_FoV_step = []
        LB_smoothness_adjacency_FoV_step = []
        LB_smoothness_outside_FoV_step = []
        LB_bitrate_level_number_step = []

        Buffer_rebuffer_step = []
        Buffer_bitrate_step = []
        Buffer_smoothness_step = []
        Buffer_reward_step = 0
        Buffer_rebuffer_time_step = []
        Buffer_FoV_inside_bitrate_step = []
        Buffer_FoV_adjacency_bitrate_step = []
        Buffer_FoV_outside_bitrate_step = []
        Buffer_smoothness_inside_FoV_step = []
        Buffer_smoothness_adjacency_FoV_step = []
        Buffer_smoothness_outside_FoV_step = []
        Buffer_bitrate_level_number_step = []

        Throughput_rebuffer_step = []
        Throughput_bitrate_step = []
        Throughput_smoothness_step = []
        Throughput_reward_step = 0
        Throughput_rebuffer_time_step = []
        Throughput_FoV_inside_bitrate_step = []
        Throughput_FoV_adjacency_bitrate_step = []
        Throughput_FoV_outside_bitrate_step = []
        Throughput_smoothness_inside_FoV_step = []
        Throughput_smoothness_adjacency_FoV_step = []
        Throughput_smoothness_outside_FoV_step = []
        Throughput_bitrate_level_number_step = []

        BOLA_rebuffer_step = []
        BOLA_bitrate_step = []
        BOLA_smoothness_step = []
        BOLA_reward_step = 0
        BOLA_rebuffer_time_step = []
        BOLA_FoV_inside_bitrate_step = []
        BOLA_FoV_adjacency_bitrate_step = []
        BOLA_FoV_outside_bitrate_step = []
        BOLA_smoothness_inside_FoV_step = []
        BOLA_smoothness_adjacency_FoV_step = []
        BOLA_smoothness_outside_FoV_step = []

        done = False ; ep_steps = 0 ; option_termination = True ; curr_op_len = 0
        while not done and ep_steps < args.max_steps_ep:  # 视频播完或步数太大了（大于max_step_ep 1000步）
            epsilon = option_critic.epsilon

            if option_termination:
                # option_list.
                option_bitrate_list.append(np.mean(option_one_bitrate_list))
                option_smoothness_list.append(np.mean(option_one_smoothness_list))
                option_rebuffer_list.append(np.mean(option_one_rebuffer_list))
                option_one_length_list.append(curr_op_len)
                option_one_selection_list.append(current_option)

                option_lengths[current_option].append(curr_op_len)
                current_option = np.random.choice(args.num_options) if np.random.rand() < epsilon else greedy_option
                curr_op_len = 0


                option_one_bitrate_list = []
                option_one_smoothness_list = []
                option_one_rebuffer_list = []

            # print(option_termination)
            # 这里的state用6次不一样的


            # 这里是
            action, logp, entropy = option_critic.get_action(state, current_option)

            # 看当前的epi 决定action是针对哪个tile的，沿id决策 3 bitrate_level_total,smoothness_within_frame,rebuffer_event
            next_obs, reward, done, _ = env.step(action,current_option)
            buffer.push(obs, current_option, reward, next_obs, done)
            option_step.append(current_option)
            bitrate_step.append(_[0])
            smoothness_step.append(_[1])
            rebuffer_step.append(_[2])
            rebuffer_time_step.append(_[5])
            FoV_inside_bitrate_step.append(_[6])
            FoV_adjacency_bitrate_step.append(_[7])
            FoV_outside_bitrate_step.append(_[8])
            smoothness_inside_FoV_step.append(_[9])
            smoothness_adjacency_FoV_step.append(_[10])
            smoothness_outside_FoV_step.append(_[11])
            option_one_bitrate_list.append(_[0])
            option_one_smoothness_list.append(_[1])
            option_one_rebuffer_list.append(_[2])
            bitrate_level_number_step.append(_[6])
            bitrate_level_number_step[-1].extend(_[7])
            bitrate_level_number_step[-1].extend(_[8])

            HB_rebuffer_step.append(_[12][3])
            HB_bitrate_step.append(_[12][1])
            HB_smoothness_step.append(_[12][2])
            HB_reward_step += _[12][0]
            HB_rebuffer_time_step.append(_[12][4])
            HB_FoV_inside_bitrate_step.append(_[12][5])
            HB_FoV_adjacency_bitrate_step.append(_[12][6])
            HB_FoV_outside_bitrate_step.append(_[12][7])
            HB_smoothness_inside_FoV_step.append(_[12][8])
            HB_smoothness_adjacency_FoV_step.append(_[12][9])
            HB_smoothness_outside_FoV_step.append(_[12][10])
            HB_bitrate_level_number_step.append(_[12][5])
            HB_bitrate_level_number_step[-1].extend(_[12][6])
            HB_bitrate_level_number_step[-1].extend(_[12][7])

            DP_rebuffer_step.append(_[13][3])
            DP_bitrate_step.append(_[13][1])
            DP_smoothness_step.append(_[13][2])
            DP_reward_step += _[13][0]
            DP_rebuffer_time_step.append(_[13][4])
            DP_FoV_inside_bitrate_step.append(_[13][5])
            DP_FoV_adjacency_bitrate_step.append(_[13][6])
            DP_FoV_outside_bitrate_step.append(_[13][7])
            DP_smoothness_inside_FoV_step.append(_[13][8])
            DP_smoothness_adjacency_FoV_step.append(_[13][9])
            DP_smoothness_outside_FoV_step.append(_[13][10])
            DP_bitrate_level_number_step.append(_[13][5])
            DP_bitrate_level_number_step[-1].extend(_[13][6])
            DP_bitrate_level_number_step[-1].extend(_[13][7])

            LB_rebuffer_step.append(_[14][3])
            LB_bitrate_step.append(_[14][1])
            LB_smoothness_step.append(_[14][2])
            LB_reward_step += _[14][0]
            LB_rebuffer_time_step.append(_[14][4])
            LB_FoV_inside_bitrate_step.append(_[14][5])
            LB_FoV_adjacency_bitrate_step.append(_[14][6])
            LB_FoV_outside_bitrate_step.append(_[14][7])
            LB_smoothness_inside_FoV_step.append(_[14][8])
            LB_smoothness_adjacency_FoV_step.append(_[14][9])
            LB_smoothness_outside_FoV_step.append(_[14][10])
            LB_bitrate_level_number_step.append(_[14][5])
            LB_bitrate_level_number_step[-1].extend(_[14][6])
            LB_bitrate_level_number_step[-1].extend(_[14][7])

            # BOLA_rebuffer_step.append(_[15][3])
            # BOLA_bitrate_step.append(_[15][1])
            # BOLA_smoothness_step.append(_[15][2])
            # BOLA_reward_step += _[15][0]
            # BOLA_rebuffer_time_step.append(_[15][4])
            # BOLA_FoV_inside_bitrate_step.append(_[15][5])
            # BOLA_FoV_adjacency_bitrate_step.append(_[15][6])
            # BOLA_FoV_outside_bitrate_step.append(_[15][7])
            # BOLA_smoothness_inside_FoV_step.append(_[15][8])
            # BOLA_smoothness_adjacency_FoV_step.append(_[15][9])
            # BOLA_smoothness_outside_FoV_step.append(_[15][10])

            Buffer_rebuffer_step.append(_[16][3])
            Buffer_bitrate_step.append(_[16][1])
            Buffer_smoothness_step.append(_[16][2])
            Buffer_reward_step += _[16][0]
            Buffer_rebuffer_time_step.append(_[16][4])
            Buffer_FoV_inside_bitrate_step.append(_[16][5])
            Buffer_FoV_adjacency_bitrate_step.append(_[16][6])
            Buffer_FoV_outside_bitrate_step.append(_[16][7])
            Buffer_smoothness_inside_FoV_step.append(_[16][8])
            Buffer_smoothness_adjacency_FoV_step.append(_[16][9])
            Buffer_smoothness_outside_FoV_step.append(_[16][10])
            Buffer_bitrate_level_number_step.append(_[16][5])
            Buffer_bitrate_level_number_step[-1].extend(_[16][6])
            Buffer_bitrate_level_number_step[-1].extend(_[16][7])

            Throughput_rebuffer_step.append(_[17][3])
            Throughput_bitrate_step.append(_[17][1])
            Throughput_smoothness_step.append(_[17][2])
            Throughput_reward_step += _[17][0]
            Throughput_rebuffer_time_step.append(_[17][4])
            Throughput_FoV_inside_bitrate_step.append(_[17][5])
            Throughput_FoV_adjacency_bitrate_step.append(_[17][6])
            Throughput_FoV_outside_bitrate_step.append(_[17][7])
            Throughput_smoothness_inside_FoV_step.append(_[17][8])
            Throughput_smoothness_adjacency_FoV_step.append(_[17][9])
            Throughput_smoothness_outside_FoV_step.append(_[17][10])
            Throughput_bitrate_level_number_step.append(_[17][5])
            Throughput_bitrate_level_number_step[-1].extend(_[17][6])
            Throughput_bitrate_level_number_step[-1].extend(_[17][7])


            old_state = state
            state = option_critic.get_state(to_tensor(next_obs))
            # print(next_obs)
            option_termination, greedy_option = option_critic.predict_option_termination(state, current_option)
            rewards += reward

            actor_loss, critic_loss = None, None  # 这个buffer只有大于1024才做反向传播，而且大于之后一个step传一个？

            if len(buffer) > args.batch_size:
                actor_loss = actor_loss_fn(obs, current_option, \
                    reward, logp, entropy, done, next_obs, option_critic, option_critic_prime, args)
                loss = actor_loss

                if steps % args.update_frequency == 0:
                    data_batch = buffer.sample(args.batch_size)
                    critic_loss = critic_loss_fn(option_critic, option_critic_prime, data_batch, args)
                    loss1 = loss + critic_loss

                    optim.zero_grad()
                    loss1.backward()
                    optim.step()
                else:
                    optim.zero_grad()
                    loss.backward()
                    optim.step()

                if steps % args.freeze_interval == 0:
                    option_critic_prime.load_state_dict(option_critic.state_dict())

            # update global steps etc
            steps += 1  # 程序 steps 总步数
            ep_steps += 1 # 视频内总步数（考虑到视频结束与否且步数不大于1000）
            curr_op_len += 1
            obs = next_obs

            logger.log_data(steps, reward , actor_loss, critic_loss, epsilon)
            if steps % 10000 == 0:
                torch.save(option_critic.state_dict(),MODEL_PATH)

        reward_list.append(rewards/ep_steps)
        option_list.append(option_step)
        bitrate_epi.append(np.mean(bitrate_step))
        smoothness_epi.append(np.mean(smoothness_step))
        rebuffer_epi.append(np.sum(rebuffer_step))
        rebuffer_time_epi.append(np.sum(rebuffer_time_step))
        FoV_inside_bitrate_epi.append(np.mean(FoV_inside_bitrate_step))
        FoV_adjacency_bitrate_epi.append(np.mean(FoV_adjacency_bitrate_step))
        FoV_outside_bitrate_epi.append(np.mean(FoV_outside_bitrate_step))
        smoothness_inside_FoV.append(np.mean(smoothness_inside_FoV_step))
        smoothness_adjacency_FoV.append(np.mean(smoothness_adjacency_FoV_step))
        smoothness_outside_FoV.append(np.mean(smoothness_outside_FoV_step))
        bitrate_level_number_epi.append(bitrate_level_number_step)

        DP_rebuffer_epi.append(np.sum(DP_rebuffer_step))
        DP_bitrate_epi.append(np.mean(DP_bitrate_step))
        DP_smoothness_epi.append(np.mean(DP_smoothness_step))
        DP_reward_list.append(DP_reward_step/ep_steps)
        DP_rebuffer_time_epi.append(np.sum(DP_rebuffer_time_step))
        DP_FoV_inside_bitrate_epi.append(np.mean(DP_FoV_inside_bitrate_step))
        DP_FoV_adjacency_bitrate_epi.append(np.mean(DP_FoV_adjacency_bitrate_step))
        DP_FoV_outside_bitrate_epi.append(np.mean(DP_FoV_outside_bitrate_step))
        DP_smoothness_inside_FoV.append(np.mean(DP_smoothness_inside_FoV_step))
        DP_smoothness_adjacency_FoV.append(np.mean(DP_smoothness_adjacency_FoV_step))
        DP_smoothness_outside_FoV.append(np.mean(DP_smoothness_outside_FoV_step))
        DP_bitrate_level_number_epi.append(DP_bitrate_level_number_step)

        HB_rebuffer_epi.append(np.sum(HB_rebuffer_step))
        HB_bitrate_epi.append(np.mean(HB_bitrate_step))
        HB_smoothness_epi.append(np.mean(HB_smoothness_step))
        HB_reward_list.append(HB_reward_step/ep_steps)
        HB_rebuffer_time_epi.append(np.sum(HB_rebuffer_time_step))
        HB_FoV_inside_bitrate_epi.append(np.mean(HB_FoV_inside_bitrate_step))
        HB_FoV_adjacency_bitrate_epi.append(np.mean(HB_FoV_adjacency_bitrate_step))
        HB_FoV_outside_bitrate_epi.append(np.mean(HB_FoV_outside_bitrate_step))
        HB_smoothness_inside_FoV.append(np.mean(HB_smoothness_inside_FoV_step))
        HB_smoothness_adjacency_FoV.append(np.mean(HB_smoothness_adjacency_FoV_step))
        HB_smoothness_outside_FoV.append(np.mean(HB_smoothness_outside_FoV_step))
        HB_bitrate_level_number_epi.append(HB_bitrate_level_number_step)

        LB_rebuffer_epi.append(np.sum(LB_rebuffer_step).astype(np.float))
        LB_bitrate_epi.append(np.mean(LB_bitrate_step))
        LB_smoothness_epi.append(np.mean(LB_smoothness_step))
        LB_reward_list.append(LB_reward_step/ep_steps)
        LB_rebuffer_time_epi.append(np.sum(LB_rebuffer_time_step))
        LB_FoV_inside_bitrate_epi.append(np.mean(LB_FoV_inside_bitrate_step))
        LB_FoV_adjacency_bitrate_epi.append(np.mean(LB_FoV_adjacency_bitrate_step))
        LB_FoV_outside_bitrate_epi.append(np.mean(LB_FoV_outside_bitrate_step))
        LB_smoothness_inside_FoV.append(np.mean(LB_smoothness_inside_FoV_step))
        LB_smoothness_adjacency_FoV.append(np.mean(LB_smoothness_adjacency_FoV_step))
        LB_smoothness_outside_FoV.append(np.mean(LB_smoothness_outside_FoV_step))
        LB_bitrate_level_number_epi.append(LB_bitrate_level_number_step)

        Buffer_rebuffer_epi.append(np.sum(Buffer_rebuffer_step).astype(np.float))
        Buffer_bitrate_epi.append(np.mean(Buffer_bitrate_step))
        Buffer_smoothness_epi.append(np.mean(Buffer_smoothness_step))
        Buffer_reward_list.append(Buffer_reward_step/ep_steps)
        Buffer_rebuffer_time_epi.append(np.sum(Buffer_rebuffer_time_step))
        Buffer_FoV_inside_bitrate_epi.append(np.mean(Buffer_FoV_inside_bitrate_step))
        Buffer_FoV_adjacency_bitrate_epi.append(np.mean(Buffer_FoV_adjacency_bitrate_step))
        Buffer_FoV_outside_bitrate_epi.append(np.mean(Buffer_FoV_outside_bitrate_step))
        Buffer_smoothness_inside_FoV.append(np.mean(Buffer_smoothness_inside_FoV_step))
        Buffer_smoothness_adjacency_FoV.append(np.mean(Buffer_smoothness_adjacency_FoV_step))
        Buffer_smoothness_outside_FoV.append(np.mean(Buffer_smoothness_outside_FoV_step))
        Buffer_bitrate_level_number_epi.append(Buffer_bitrate_level_number_step)

        Throughput_rebuffer_epi.append(np.sum(Throughput_rebuffer_step).astype(np.float))
        Throughput_bitrate_epi.append(np.mean(Throughput_bitrate_step))
        Throughput_smoothness_epi.append(np.mean(Throughput_smoothness_step))
        Throughput_reward_list.append(Throughput_reward_step/ep_steps)
        Throughput_rebuffer_time_epi.append(np.sum(Throughput_rebuffer_time_step))
        Throughput_FoV_inside_bitrate_epi.append(np.mean(Throughput_FoV_inside_bitrate_step))
        Throughput_FoV_adjacency_bitrate_epi.append(np.mean(Throughput_FoV_adjacency_bitrate_step))
        Throughput_FoV_outside_bitrate_epi.append(np.mean(Throughput_FoV_outside_bitrate_step))
        Throughput_smoothness_inside_FoV.append(np.mean(Throughput_smoothness_inside_FoV_step))
        Throughput_smoothness_adjacency_FoV.append(np.mean(Throughput_smoothness_adjacency_FoV_step))
        Throughput_smoothness_outside_FoV.append(np.mean(Throughput_smoothness_outside_FoV_step))
        Throughput_bitrate_level_number_epi.append(Throughput_bitrate_level_number_step)
        # BOLA_rebuffer_epi.append(np.sum(BOLA_rebuffer_step))
        # BOLA_bitrate_epi.append(np.mean(BOLA_bitrate_step))
        # BOLA_smoothness_epi.append(np.mean(BOLA_smoothness_step))
        # BOLA_reward_list.append(BOLA_reward_step/ep_steps)
        # BOLA_rebuffer_time_epi.append(np.sum(BOLA_rebuffer_time_step))
        # BOLA_FoV_inside_bitrate_epi.append(np.mean(BOLA_FoV_inside_bitrate_step))
        # BOLA_FoV_adjacency_bitrate_epi.append(np.mean(BOLA_FoV_adjacency_bitrate_step))
        # BOLA_FoV_outside_bitrate_epi.append(np.mean(BOLA_FoV_outside_bitrate_step))
        # BOLA_smoothness_inside_FoV.append(np.mean(BOLA_smoothness_inside_FoV_step))
        # BOLA_smoothness_adjacency_FoV.append(np.mean(BOLA_smoothness_adjacency_FoV_step))
        # BOLA_smoothness_outside_FoV.append(np.mean(BOLA_smoothness_outside_FoV_step))

        logger.log_episode(steps, rewards, option_lengths, ep_steps, epsilon)

    plt.figure(2)
    x = range(len(reward_list))
    plt.plot(x, reward_list)
    plt.title(
        "reward_epi_list (avg)" + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(round(time.time() * 1000)) / 1000)))
    plt.savefig('./figure1.png')  # 保存图片
    plt.show()

    # plt.figure(3)
    # x = range(len(option_list))
    # plt.plot(x, option_list)
    # plt.title(
    #     "option_selection_list" + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(round(time.time() * 1000)) / 1000)))
    # plt.savefig('./figure2.png')  # 保存图片
    # plt.show()

    import scipy.io as sio
    mat_name = "data" + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(round(time.time() * 1000)) / 1000)) + ".mat"
    mat_name = mat_name.replace(' ','_')
    mat_name = mat_name.replace('-','_')
    mat_name = mat_name.replace(':', '_')
    #                           "BOLA_FoV_inside_bitrate_epi": BOLA_FoV_inside_bitrate_epi,
    #                           "BOLA_FoV_adjacency_bitrate_epi": BOLA_FoV_adjacency_bitrate_epi,
    #                           "BOLA_FoV_outside_bitrate_epi": BOLA_FoV_outside_bitrate_epi,
    #                           "BOLA_smoothness_inside_FoV": BOLA_smoothness_inside_FoV,
    #                           "BOLA_smoothness_adjacency_FoV": BOLA_smoothness_adjacency_FoV,
    #                           "BOLA_smoothness_outside_FoV": BOLA_smoothness_outside_FoV,
    #                           "BOLA_rebuffer_epi": BOLA_rebuffer_epi,
    #                           "BOLA_bitrate_epi": BOLA_bitrate_epi,
    #                           "BOLA_smoothness_epi": BOLA_smoothness_epi,
    #                           "BOLA_reward_list": BOLA_reward_list,
    #                           "BOLA_rebuffer_time_epi": BOLA_rebuffer_time_epi,
    sio.savemat(mat_name,{"reward_list": reward_list,
                          "option_list":option_list,
                            "bitrate_epi":bitrate_epi,
                          "smoothness_epi":smoothness_epi,
                          "rebuffer_epi":rebuffer_epi,
                          "DP_rebuffer_epi":DP_rebuffer_epi,
                          "DP_bitrate_epi":DP_bitrate_epi,
                          "DP_smoothness_epi":DP_smoothness_epi,
                          "DP_reward_list":DP_reward_list,
                          "HB_rebuffer_epi":HB_rebuffer_epi,
                          "HB_bitrate_epi":HB_bitrate_epi,
                          "HB_smoothness_epi":HB_smoothness_epi,
                          "HB_reward_list":HB_reward_list,
                          "LB_rebuffer_epi": LB_rebuffer_epi,
                          "LB_bitrate_epi": LB_bitrate_epi,
                          "LB_smoothness_epi": LB_smoothness_epi,
                          "LB_reward_list": LB_reward_list,
                          "FoV_inside_bitrate_epi":FoV_inside_bitrate_epi,
                          "FoV_adjacency_bitrate_epi":FoV_adjacency_bitrate_epi,
                          "FoV_outside_bitrate_epi":FoV_outside_bitrate_epi,
                          "smoothness_inside_FoV":smoothness_inside_FoV,
                          "smoothness_adjacency_FoV":smoothness_adjacency_FoV,
                          "smoothness_outside_FoV":smoothness_outside_FoV,
                          "DP_FoV_inside_bitrate_epi": DP_FoV_inside_bitrate_epi,
                          "DP_FoV_adjacency_bitrate_epi": DP_FoV_adjacency_bitrate_epi,
                          "DP_FoV_outside_bitrate_epi": DP_FoV_outside_bitrate_epi,
                          "DP_smoothness_inside_FoV": DP_smoothness_inside_FoV,
                          "DP_smoothness_adjacency_FoV": DP_smoothness_adjacency_FoV,
                          "DP_smoothness_outside_FoV": DP_smoothness_outside_FoV,
                          "HB_FoV_inside_bitrate_epi": HB_FoV_inside_bitrate_epi,
                          "HB_FoV_adjacency_bitrate_epi": HB_FoV_adjacency_bitrate_epi,
                          "HB_FoV_outside_bitrate_epi": HB_FoV_outside_bitrate_epi,
                          "HB_smoothness_inside_FoV": HB_smoothness_inside_FoV,
                          "HB_smoothness_adjacency_FoV": HB_smoothness_adjacency_FoV,
                          "HB_smoothness_outside_FoV": HB_smoothness_outside_FoV,
                          "LB_FoV_inside_bitrate_epi": LB_FoV_inside_bitrate_epi,
                          "LB_FoV_adjacency_bitrate_epi": LB_FoV_adjacency_bitrate_epi,
                          "LB_FoV_outside_bitrate_epi": LB_FoV_outside_bitrate_epi,
                          "LB_smoothness_inside_FoV": LB_smoothness_inside_FoV,
                          "LB_smoothness_adjacency_FoV": LB_smoothness_adjacency_FoV,
                          "LB_smoothness_outside_FoV": LB_smoothness_outside_FoV,
                          "rebuffer_time_epi":rebuffer_time_epi,
                          "DP_rebuffer_time_epi": DP_rebuffer_time_epi,
                          "HB_rebuffer_time_epi": HB_rebuffer_time_epi,
                          "LB_rebuffer_time_epi": LB_rebuffer_time_epi,
                          "option_one_length_list":option_one_length_list,
                          "option_one_selection_list":option_one_selection_list,
                          "option_one_bitrate_list":option_one_bitrate_list,
                          "option_one_smoothness_list":option_one_smoothness_list,
                          "option_one_rebuffer_list":option_one_rebuffer_list,
                          "option_bitrate_list":option_bitrate_list,
                          "option_rebuffer_list":option_rebuffer_list,
                          "option_smoothness_list":option_smoothness_list,
                          "Buffer_rebuffer_epi": Buffer_rebuffer_epi,
                          "Buffer_bitrate_epi": Buffer_bitrate_epi,
                          "Buffer_smoothness_epi": Buffer_smoothness_epi,
                          "Buffer_reward_list": Buffer_reward_list,
                          "Buffer_FoV_inside_bitrate_epi": Buffer_FoV_inside_bitrate_epi,
                          "Buffer_FoV_adjacency_bitrate_epi": Buffer_FoV_adjacency_bitrate_epi,
                          "Buffer_FoV_outside_bitrate_epi": Buffer_FoV_outside_bitrate_epi,
                          "Buffer_smoothness_inside_FoV": Buffer_smoothness_inside_FoV,
                          "Buffer_smoothness_adjacency_FoV": Buffer_smoothness_adjacency_FoV,
                          "Buffer_smoothness_outside_FoV": Buffer_smoothness_outside_FoV,
                          "Buffer_rebuffer_time_epi": Buffer_rebuffer_time_epi,
                          "Throughput_rebuffer_epi": Throughput_rebuffer_epi,
                          "Throughput_bitrate_epi": Throughput_bitrate_epi,
                          "Throughput_smoothness_epi": Throughput_smoothness_epi,
                          "Throughput_reward_list": Throughput_reward_list,
                          "Throughput_FoV_inside_bitrate_epi": Throughput_FoV_inside_bitrate_epi,
                          "Throughput_FoV_adjacency_bitrate_epi": Throughput_FoV_adjacency_bitrate_epi,
                          "Throughput_FoV_outside_bitrate_epi": Throughput_FoV_outside_bitrate_epi,
                          "Throughput_smoothness_inside_FoV": Throughput_smoothness_inside_FoV,
                          "Throughput_smoothness_adjacency_FoV": Throughput_smoothness_adjacency_FoV,
                          "Throughput_smoothness_outside_FoV": Throughput_smoothness_outside_FoV,
                          "Throughput_rebuffer_time_epi": Throughput_rebuffer_time_epi,
                          "bitrate_level_number_epi": bitrate_level_number_epi,
                          "DP_bitrate_level_number_epi": DP_bitrate_level_number_epi,
                          "HB_bitrate_level_number_epi": HB_bitrate_level_number_epi,
                          "LB_bitrate_level_number_epi": LB_bitrate_level_number_epi,
                          "Buffer_bitrate_level_number_epi": Buffer_bitrate_level_number_epi,
                          "Throughput_bitrate_level_number_epi": Throughput_bitrate_level_number_epi
                        })


if __name__=="__main__":
    times = 10;
    for i in range(times):
        args = parser.parse_args()
        run(args)
